<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>全栈进阶那些事 - 一只爱技术的程序猿，想把分享变成一种习惯！ </title><meta name="author" content="liuyuantao"><meta name="copyright" content="liuyuantao"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="一个有趣、有料、有内涵的地方!">
<meta property="og:type" content="website">
<meta property="og:title" content="全栈进阶那些事">
<meta property="og:url" content="https://liuyuantao.gitee.io/index.html">
<meta property="og:site_name" content="全栈进阶那些事">
<meta property="og:description" content="一个有趣、有料、有内涵的地方!">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://liuyuantao.gitee.io/img/zhiyao_avatar.jpg">
<meta property="article:author" content="liuyuantao">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://liuyuantao.gitee.io/img/zhiyao_avatar.jpg"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="https://liuyuantao.gitee.io/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//hm.baidu.com"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css" media="print" onload="this.media='all'"><script>var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?d43383137144f76a37f1074576bc674e";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: {"path":"search.xml","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: {"defaultEncoding":1,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"简"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":200},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: true,
    post: true
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: {"limitCount":500,"languages":{"author":"作者: liuyuantao","link":"链接: ","source":"来源: 全栈进阶那些事","info":"著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。"}},
  lightbox: 'mediumZoom',
  Snackbar: {"chs_to_cht":"你已切换为繁体","cht_to_chs":"你已切换为简体","day_to_night":"你已切换为深色模式","night_to_day":"你已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#2d3035","position":"top-right"},
  source: {
    jQuery: 'https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js',
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
    },
    fancybox: {
      js: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.js',
      css: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.css'
    }
  },
  isPhotoFigcaption: true,
  islazyload: false,
  isanchor: true
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '全栈进阶那些事',
  isPost: false,
  isHome: true,
  isHighlightShrink: false,
  isToc: false,
  postUpdate: '2021-11-23 18:55:57'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const fontSizeVal = saveToLocal.get('global-font-size')
    if (fontSizeVal !== undefined) {
      document.documentElement.style.setProperty('--global-font-size', fontSizeVal + 'px')
    }
    
    const detectApple = () => {
      if (GLOBAL_CONFIG_SITE.isHome && /iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><meta name="generator" content="Hexo 5.4.0"><link rel="alternate" href="/atom.xml" title="全栈进阶那些事" type="application/atom+xml">
</head><body><div id="loading-box"><div class="loading-left-bg"></div><div class="loading-right-bg"></div><div class="spinner-box"><div class="configure-border-1"><div class="configure-core"></div></div><div class="configure-border-2"><div class="configure-core"></div></div><div class="loading-word">加载中...</div></div></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/img/zhiyao_avatar.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/archives/"><div class="headline">文章</div><div class="length-num">41</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/tags/"><div class="headline">标签</div><div class="length-num">14</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="page" id="body-wrap"><header class="not-top-img" id="page-header"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">全栈进阶那些事</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav></header><main class="layout" id="content-inner"><div class="recent-posts" id="recent-posts"><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2021/11/19/4e544b17c3ac.html" title="Nginx日志按天自动切割">     <img class="post_bg" src="https://cdn.pixabay.com/photo/2021/10/25/17/16/nature-6741602__340.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Nginx日志按天自动切割"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/11/19/4e544b17c3ac.html" title="Nginx日志按天自动切割">Nginx日志按天自动切割</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-11-19T09:40:06.000Z" title="发表于 2021-11-19 17:40:06">2021-11-19</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E8%BF%90%E7%BB%B4/">运维</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Nginx/">Nginx</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/%E8%BF%90%E7%BB%B4/">运维</a></span></div><div class="content">
使用Nginx做反向代理，代理了3台服务器，由于所有的访问都经过Nginx，发现access日志增长的很快，时间长了，日志文件难免会变得很大，打开查看很不方便，而且会影响服务器性能，因此有必要对nginx日志进行每日切割保存，减小文件大小，同时也更方便查看日志。

实现思路shell脚本+定时任务+nginx信号控制，完成日志定时切割。
具体步骤在需要保存日志或者是其他目录，新建一个shell脚本。
vim /usr/local/openresty/nginx/logs/log_rotate.sh
编辑内容如下：
#!/bin/bash#安装目录下日志文件base_log_path=&#x27;/usr/local/openresty/nginx/logs/access.log&#x27;base_error_path=&#x27;/usr/local/openresty/nginx/logs/error.log&#x27;#需要保存的目录位置log_path=&#x27;/data_lytdev_dir/nginx/logs/&#x27;#获取月份log_month=$(date  ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2021/11/18/b4485f4970ce.html" title="ClickHouse新特性之SQL化用户配置">     <img class="post_bg" src="https://cdn.pixabay.com/photo/2017/01/31/18/21/books-2026194__340.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="ClickHouse新特性之SQL化用户配置"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/11/18/b4485f4970ce.html" title="ClickHouse新特性之SQL化用户配置">ClickHouse新特性之SQL化用户配置</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-11-18T05:55:38.000Z" title="发表于 2021-11-18 13:55:38">2021-11-18</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/ClickHouse/">ClickHouse</a></span></div><div class="content">在之前的ClickHouse版本中，我们只能通过修改users.xml文件来配置用户及相关的参数（权限、资源限制、查询配额等），不是很方便。好在从20.5版本起，ClickHouse开始支持SQL化的用户配置（如同MySQL、PostgreSQL一样），易用性增强了很多。
SQL化用户配置默认是关闭的，要启用它，需要在users.xml中的一个用户（一般就是默认的default）下添加：
&lt;access_management&gt;1&lt;/access_management&gt;

另外还需要在config.xml中配置权限管理数据的存储位置：
&lt;access_control_path&gt;/data1/clickhouse/access/&lt;/access_control_path&gt;

这样我们就可以用default用户管理其他用户了。ClickHouse采用与MySQL近似的RBAC（Role-based access control，基于角色的访问控制）机制，所以语法也很像。下面简要列举几个操作，更加详细的内容可以参见官方文档：https://cli ...</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2021/11/18/6d7187a575ff.html" title="MergeTree引擎的固定/自适应索引粒度">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-8479e16ac3738d1e8243838c11c9e96aba9.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="MergeTree引擎的固定/自适应索引粒度"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/11/18/6d7187a575ff.html" title="MergeTree引擎的固定/自适应索引粒度">MergeTree引擎的固定/自适应索引粒度</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-11-18T05:52:18.000Z" title="发表于 2021-11-18 13:52:18">2021-11-18</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/ClickHouse/">ClickHouse</a></span></div><div class="content">前言
我们在刚开始学习ClickHouse的MergeTree引擎时，建表语句的末尾总会有SETTINGS index_granularity = 8192这句话（其实不写也可以），表示索引粒度为8192。在每个data part中，索引粒度参数的含义有二：


每隔index_granularity行对主键组的数据进行采样，形成稀疏索引，并存储在primary.idx文件中；
每隔index_granularity行对每一列的压缩数据（[column].bin）进行采样，形成数据标记，并存储在[column].mrk文件中。

index_granularity、primary.idx、[column].bin/mrk之间的关系可以用下面的简图来表示。

但是早在一年前的ClickHouse 19.11.8版本，社区就引入了自适应（adaptive）索引粒度的特性，并且在当前的版本中是默认开启的。也就是说，主键索引和数据标记生成的间隔可以不再固定，更加灵活。下面通过实例来讲解固定索引粒度和自适应索引粒度之间的不同之处。
固定索引粒度利用Yandex.Metrica提供的hits_v1 ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2021/11/18/23f3f36da036.html" title="无缝更改ClickHouse物化视图SELECT逻辑的方法">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-ac406843f42bfac15c16815aeb376bca9ad.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="无缝更改ClickHouse物化视图SELECT逻辑的方法"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/11/18/23f3f36da036.html" title="无缝更改ClickHouse物化视图SELECT逻辑的方法">无缝更改ClickHouse物化视图SELECT逻辑的方法</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-11-18T05:43:50.000Z" title="发表于 2021-11-18 13:43:50">2021-11-18</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/ClickHouse/">ClickHouse</a></span></div><div class="content">在我司的ClickHouse DWS层有一张ReplicatedAggregatingMergeTree引擎的物化视图，为近线推荐业务提供关键用户行为的计数值。该物化视图的底表（即所谓“inner”表）有两张，分别从不同的事实表聚合数据，如下图所示。 Q：算法同学希望在物化视图中增加一些用户行为，如何在保证不影响线上业务（不删表）的前提下把这些行为加进去？ A：以mv_recommend_access_stat为例。

 使用DETACH TABLE语句将底表解挂，即暂时屏蔽掉它的元数据

DETACH TABLE rtdw_dws.mv_recommend_access_stat ON CLUSTER sht_ck_cluster_pro;


 定位到它的元数据文件所在的路径，可以从系统表查询。

SELECT metadata_path FROM system.tables WHERE name = &#x27;mv_recommend_access_stat&#x27;;┌─metadata_path─────────────────────────────────────── ...</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2021/11/18/fec2d399fcb4.html" title="物化视图简介与ClickHouse中的应用示例">     <img class="post_bg" src="https://upload-images.jianshu.io/upload_images/195230-462138b5b8e6e815.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="物化视图简介与ClickHouse中的应用示例"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/11/18/fec2d399fcb4.html" title="物化视图简介与ClickHouse中的应用示例">物化视图简介与ClickHouse中的应用示例</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-11-18T05:31:13.000Z" title="发表于 2021-11-18 13:31:13">2021-11-18</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/ClickHouse/">ClickHouse</a></span></div><div class="content">前言
最近在搞520大促的事情，忙到脚不点地，所以就写些简单省事的吧。物化视图概念
我们都知道，数据库中的视图（view）是从一张或多张数据库表查询导出的虚拟表，反映基础表中数据的变化，且本身不存储数据。那么物化视图（materialized view）是什么呢？英文维基中给出的描述是相当准确的，抄录如下。

   In computing, a materialized view is a database object that contains the results of a query. For example, it may be a local copy of data located remotely, or may be a subset of the rows and/or columns of a table or join result, or may be a summary using an aggregate function.


The process of setting up a materialized view is sometimes cal ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2021/11/13/3f0a31ee0039.html" title="深入浅出学习Hive">     <img class="post_bg" src="https://img-blog.csdnimg.cn/20200529093005305.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="深入浅出学习Hive"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/11/13/3f0a31ee0039.html" title="深入浅出学习Hive">深入浅出学习Hive</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-11-13T00:36:02.000Z" title="发表于 2021-11-13 08:36:02">2021-11-13</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/Hive/">Hive</a></span></div><div class="content">
本文是基于CentOS 7.9系统环境，进行hive的学习和使用

一、Hive的简介1.1 Hive基本概念(1) 什么是hive
Hive是用于解决海量结构化日志的数据统计工具，是基于Hadoop的一个数据仓库工具，可以将结构化的数据文件映射为一张表，并提供类SQL查询功能(2) Hive的本质
Hive的本质就是将HQL转化成MapReduce程序
1.2 Hive优缺点(1) 优点

 操作接口采用类SQL语法，提供快速开发的能力（简单、容易）
 避免写MapReduce程序，减少开发人员的学习成本
 Hive优势在于处理大数据，常用于数据分析，适用于实时性要求不高的场景
 hive支持用户自定义函数，用户可以根据自己的需求来实现自己的函数

(2) 缺点

 Hive执行延迟比较高，对于处理小数据没有优势
 hive的HQL表达能力有限（迭代式算法无法表达；数据挖掘方面不擅长，由于MapReduce数据处理流程的限制，效率更高的算法却无法实现）
 hive的效率比较低（hive自动生成的MapReduce作业，通常情况下不够智能化；hive调优比较困难，粒度较粗）

1.3 ...</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2021/11/12/56ac102284d9.html" title="深入浅出学习MapReduce">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/20200617165336828.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="深入浅出学习MapReduce"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/11/12/56ac102284d9.html" title="深入浅出学习MapReduce">深入浅出学习MapReduce</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-11-12T13:58:21.000Z" title="发表于 2021-11-12 21:58:21">2021-11-12</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/MapReduce/">MapReduce</a></span></div><div class="content">
本文是基于CentOS 7.3系统环境，进行MapReduce的学习和使用本文是基于CentOS 7.3系统环境，进行MapReduce的学习和使用

1. MapReduce简介1.1 MapReduce定义MapReduce是一个分布式运算程序的编程框架，是基于Hadoop的数据分析计算的核心框架
1.2 MapReduce处理过程主要分为两个阶段：Map和Reduce

Map负责把一个任务分解成多个任务
Reduce负责把分解后多任务处理的结果进行汇总

1.3 MapReduce的优点
MapReduce易于编程只需要实现一些简单接口，就可以完成一个分布式程序，这个分布式程序可以分布到大量廉价的PC机器上运行。也就是说你写一个分布式程序，就跟写一个简单的串行程序是一模一样的。 
良好的扩展性（hadoop的特点）当你的计算资源不能满足的时候，你可以通过简单的增加机器（nodemanager）来扩展它的计算能力 
高容错性MapReduce设计的初衷就是使程序能够部署在廉价的PC机器上，这就要求它具有很高的容错性，比如其中一台机器挂了，它可以把上面的计算任务转移到另外一个节点 ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2021/11/12/2d8833eb4290.html" title="一文彻底搞懂HBase">     <img class="post_bg" src="https://img-blog.csdnimg.cn/20200715092910375.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="一文彻底搞懂HBase"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/11/12/2d8833eb4290.html" title="一文彻底搞懂HBase">一文彻底搞懂HBase</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-11-12T09:55:16.000Z" title="发表于 2021-11-12 17:55:16">2021-11-12</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/HBase/">HBase</a></span></div><div class="content">
本文是基于CentOS 7.9系统环境，进行HBase的学习和使用

一、HBase的简介1.1 HBase基本概念HBase是一种分布式、可扩展、支持海量数据存储的NoSQL数据库，可以解决HDFS随机写的问题
1.2 HBase数据模型逻辑上，HBase的数据模型同关系型数据库很类似，数据存储在一张表中，有行有列。但从HBase的底层物理存储结构（K-V）来看，HBase更像是一个multi-dimensional map

Name Space
  命名空间，类似于关系型数据库的DatabBase概念，每个命名空间下有多个表。HBase有两个自带的命名空间，分别是hbase和default，hbase中存放的是HBase内置的表，default表是用户默认使用的命名空间。

Region
  类似于关系型数据库的表概念。不同的是，HBase定义表时只需要声明列族即可，不需要声明具体的列。这意味着，往HBase写入数据时，字段可以动态、按需指定。因此，和关系型数据库相比，HBase能够轻松应对字段变更的场景。

Row
  HBase表中的每行数据都由一个RowKey和多个Col ...</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2021/10/12/245dd8355f34.html" title="一文彻底搞懂Kafka">     <img class="post_bg" src="https://img-blog.csdnimg.cn/20200620092054385.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="一文彻底搞懂Kafka"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/10/12/245dd8355f34.html" title="一文彻底搞懂Kafka">一文彻底搞懂Kafka</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-10-12T07:45:59.000Z" title="发表于 2021-10-12 15:45:59">2021-10-12</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E4%B8%AD%E9%97%B4%E4%BB%B6/">中间件</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/%E4%B8%AD%E9%97%B4%E4%BB%B6/">中间件</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/Kafka/">Kafka</a></span></div><div class="content">
本文是基于CentOS 7.9系统环境，进行Kafka的学习和使用

一、Kafka的简介1.1 Kafka基本概念(1) 什么是Kafka
Kafka是一个分布式的基于发布/订阅模式的消息队列，主要应用于大数据实时处理领域(2) 消息队列

点对点模式的消息队列

对一个消息而言，只会有一个消费者可以消费，消费者主动拉取消息，消息收到后，会将消息删除

基于发布/订阅模式的消息队列

发布到topic的消息会被所有订阅者消费，消费者消费完消息后不会删除

消息队列主动推送

适用于消费者处理能力高，生产者生产消息较少的场景，整个系统的消息处理速度由消息队列的推送速度决定，多个消费者之间处理消息的能力不同，如果消息队列按照一定的速度推送消息，可能会造成某些消费者处理消息处理不过来，某些消费者处理速度很快一直处于空闲

消费者主动拉取

适用于生产者生产消息的速度高于消费者，整个系统的消息处理速度由消费者的拉取速度决定，如果消息队列中消息较少，消费者会一直维护着一个长轮询去访问消息队列是否有消息
(3) Kafka的作用

解耦和异步

将强依赖的两个数据上下游系统，通过消息队列进行解 ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2021/09/12/a99eab3d4a13.html" title="一文彻底搞懂Flume">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/20200617165336828.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="一文彻底搞懂Flume"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/09/12/a99eab3d4a13.html" title="一文彻底搞懂Flume">一文彻底搞懂Flume</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-09-12T05:15:11.000Z" title="发表于 2021-09-12 13:15:11">2021-09-12</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/Flume/">Flume</a></span></div><div class="content">本文是基于CentOS 7.9系统环境，进行Flume的学习和使用
一、Flume的简介1.1 Flume基本概念(1) 什么是Flume
Flume是Cloudera提供的一个高可用的，高可靠的，分布式的海量日志采集、聚合和传输的系统。(2) Flume的目的
Flume最主要的作业就是，实时读取服务器本地磁盘的数据，将数据写入HDFS
1.2 Flume基本组件(1) Flume工作流程
Source采集数据并包装成Event，并将Event缓存在Channel中，Sink不断地从Channel获取Event，并解决成数据，最终将数据写入存储或索引系统(2) Agent
Agent是一个JVM进程，它以事件的形式将数据从源头送至目的。Agent主要有3个部分组成，Source、Channel、Sink(3) Source
Source是负责接收数据到Flume Agent的组件，采集数据并包装成Event。Source组件可以处理各种类型、各种格式的日志数据，包括avro、thrift、exec、jms、spooling directory、netcat、sequence gene ...</div></div></div><nav id="pagination"><div class="pagination"><span class="page-number current">1</span><a class="page-number" href="/page/2/#content-inner">2</a><span class="space">&hellip;</span><a class="page-number" href="/page/5/#content-inner">5</a><a class="extend next" rel="next" href="/page/2/#content-inner"><i class="fas fa-chevron-right fa-fw"></i></a></div></nav></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="/img/zhiyao_avatar.jpg" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">liuyuantao</div><div class="author-info__description">一个有趣、有料、有内涵的地方!</div></div><div class="card-info-data"><div class="card-info-data-item is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">41</div></a></div><div class="card-info-data-item is-center"><a href="/tags/"><div class="headline">标签</div><div class="length-num">14</div></a></div><div class="card-info-data-item is-center"><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div></div></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn card-announcement-animation"></i><span>公告</span></div><div class="announcement_content"><p><strong style="color:#858585;font-size:20px;">扫描二维码,关注我们</strong><br/><span style="color:#49b1f5;">了解更多全栈技能!</span></p><img src="/img/qrcode_for_gh_da91cebd00cc_258.jpg"></div></div><div class="sticky_layout"><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/2021/11/19/4e544b17c3ac.html" title="Nginx日志按天自动切割"><img src="https://cdn.pixabay.com/photo/2021/10/25/17/16/nature-6741602__340.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Nginx日志按天自动切割"/></a><div class="content"><a class="title" href="/2021/11/19/4e544b17c3ac.html" title="Nginx日志按天自动切割">Nginx日志按天自动切割</a><time datetime="2021-11-19T09:40:06.000Z" title="发表于 2021-11-19 17:40:06">2021-11-19</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/b4485f4970ce.html" title="ClickHouse新特性之SQL化用户配置"><img src="https://cdn.pixabay.com/photo/2017/01/31/18/21/books-2026194__340.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="ClickHouse新特性之SQL化用户配置"/></a><div class="content"><a class="title" href="/2021/11/18/b4485f4970ce.html" title="ClickHouse新特性之SQL化用户配置">ClickHouse新特性之SQL化用户配置</a><time datetime="2021-11-18T05:55:38.000Z" title="发表于 2021-11-18 13:55:38">2021-11-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/6d7187a575ff.html" title="MergeTree引擎的固定/自适应索引粒度"><img src="https://oscimg.oschina.net/oscnet/up-8479e16ac3738d1e8243838c11c9e96aba9.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="MergeTree引擎的固定/自适应索引粒度"/></a><div class="content"><a class="title" href="/2021/11/18/6d7187a575ff.html" title="MergeTree引擎的固定/自适应索引粒度">MergeTree引擎的固定/自适应索引粒度</a><time datetime="2021-11-18T05:52:18.000Z" title="发表于 2021-11-18 13:52:18">2021-11-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/23f3f36da036.html" title="无缝更改ClickHouse物化视图SELECT逻辑的方法"><img src="https://oscimg.oschina.net/oscnet/up-ac406843f42bfac15c16815aeb376bca9ad.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="无缝更改ClickHouse物化视图SELECT逻辑的方法"/></a><div class="content"><a class="title" href="/2021/11/18/23f3f36da036.html" title="无缝更改ClickHouse物化视图SELECT逻辑的方法">无缝更改ClickHouse物化视图SELECT逻辑的方法</a><time datetime="2021-11-18T05:43:50.000Z" title="发表于 2021-11-18 13:43:50">2021-11-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/fec2d399fcb4.html" title="物化视图简介与ClickHouse中的应用示例"><img src="https://upload-images.jianshu.io/upload_images/195230-462138b5b8e6e815.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="物化视图简介与ClickHouse中的应用示例"/></a><div class="content"><a class="title" href="/2021/11/18/fec2d399fcb4.html" title="物化视图简介与ClickHouse中的应用示例">物化视图简介与ClickHouse中的应用示例</a><time datetime="2021-11-18T05:31:13.000Z" title="发表于 2021-11-18 13:31:13">2021-11-18</time></div></div></div></div><div class="card-widget card-categories"><div class="item-headline">
            <i class="fas fa-folder-open"></i>
            <span>分类</span>
            
            </div>
            <ul class="card-category-list" id="aside-cat-list">
            <li class="card-category-list-item "><a class="card-category-list-link" href="/categories/Spring-SpringBoot/"><span class="card-category-list-name">Spring&SpringBoot</span><span class="card-category-list-count">2</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/%E4%B8%AD%E9%97%B4%E4%BB%B6/"><span class="card-category-list-name">中间件</span><span class="card-category-list-count">2</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/"><span class="card-category-list-name">大数据</span><span class="card-category-list-count">34</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/%E8%BF%90%E7%BB%B4/"><span class="card-category-list-name">运维</span><span class="card-category-list-count">1</span></a></li>
            </ul></div><div class="card-widget card-tags"><div class="item-headline"><i class="fas fa-tags"></i><span>标签</span></div><div class="card-tag-cloud"><a href="/tags/Hadoop/" style="font-size: 1.45em; color: rgb(29, 18, 195)">Hadoop</a><a href="/tags/HDFS/" style="font-size: 1.39em; color: rgb(67, 37, 146)">HDFS</a><a href="/tags/ClickHouse/" style="font-size: 1.27em; color: rgb(1, 73, 96)">ClickHouse</a><a href="/tags/Yarn/" style="font-size: 1.33em; color: rgb(150, 127, 36)">Yarn</a><a href="/tags/%E8%84%9A%E6%9C%AC/" style="font-size: 1.15em; color: rgb(88, 131, 138)">脚本</a><a href="/tags/Nginx/" style="font-size: 1.15em; color: rgb(178, 37, 116)">Nginx</a><a href="/tags/%E8%BF%90%E7%BB%B4/" style="font-size: 1.15em; color: rgb(128, 60, 172)">运维</a><a href="/tags/HBase/" style="font-size: 1.15em; color: rgb(0, 104, 140)">HBase</a><a href="/tags/%E4%B8%AD%E9%97%B4%E4%BB%B6/" style="font-size: 1.21em; color: rgb(40, 97, 23)">中间件</a><a href="/tags/Zookeeper/" style="font-size: 1.15em; color: rgb(40, 178, 154)">Zookeeper</a><a href="/tags/Hive/" style="font-size: 1.27em; color: rgb(147, 104, 22)">Hive</a><a href="/tags/Kafka/" style="font-size: 1.15em; color: rgb(176, 132, 95)">Kafka</a><a href="/tags/MapReduce/" style="font-size: 1.15em; color: rgb(157, 129, 39)">MapReduce</a><a href="/tags/Flume/" style="font-size: 1.15em; color: rgb(72, 148, 10)">Flume</a></div></div><div class="card-widget card-archives"><div class="item-headline"><i class="fas fa-archive"></i><span>归档</span></div><ul class="card-archive-list"><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/11/"><span class="card-archive-list-date">十一月 2021</span><span class="card-archive-list-count">8</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/10/"><span class="card-archive-list-date">十月 2021</span><span class="card-archive-list-count">1</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/09/"><span class="card-archive-list-date">九月 2021</span><span class="card-archive-list-count">1</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/07/"><span class="card-archive-list-date">七月 2021</span><span class="card-archive-list-count">1</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2020/11/"><span class="card-archive-list-date">十一月 2020</span><span class="card-archive-list-count">27</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2020/09/"><span class="card-archive-list-date">九月 2020</span><span class="card-archive-list-count">2</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2019/11/"><span class="card-archive-list-date">十一月 2019</span><span class="card-archive-list-count">1</span></a></li></ul></div><div class="card-widget card-webinfo"><div class="item-headline"><i class="fas fa-chart-line"></i><span>网站资讯</span></div><div class="webinfo"><div class="webinfo-item"><div class="item-name">文章数目 :</div><div class="item-count">41</div></div><div class="webinfo-item"><div class="item-name">已运行时间 :</div><div class="item-count" id="runtimeshow" data-publishDate="2020-08-31T16:00:00.000Z"></div></div><div class="webinfo-item"><div class="item-name">本站总字数 :</div><div class="item-count">59.5k</div></div><div class="webinfo-item"><div class="item-name">本站访客数 :</div><div class="item-count" id="busuanzi_value_site_uv"></div></div><div class="webinfo-item"><div class="item-name">本站总访问量 :</div><div class="item-count" id="busuanzi_value_site_pv"></div></div><div class="webinfo-item"><div class="item-name">最后更新时间 :</div><div class="item-count" id="last-push-date" data-lastPushDate="2021-11-23T10:55:57.650Z"></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2021 By liuyuantao</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div><div class="footer_custom_text">扫描二维码,关注我们!了解更多全栈技能!<br/><img style="width:100px;height:100px;" src="/img/qrcode_for_gh_da91cebd00cc_258.jpg" ></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="font-plus" type="button" title="放大字体"><i class="fas fa-plus"></i></button><button id="font-minus" type="button" title="缩小字体"><i class="fas fa-minus"></i></button><button id="translateLink" type="button" title="简繁转换">简</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><div class="search-dialog__title" id="local-search-title">本地搜索</div><div id="local-input-panel"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div></div><hr/><div id="local-search-results"></div><span class="search-close-button"><i class="fas fa-times"></i></span></div><div id="search-mask"></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/medium-zoom/dist/medium-zoom.min.js"></script><script src="https://cdn.jsdelivr.net/npm/instant.page/instantpage.min.js" type="module"></script><script src="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><script src="/js/search/local-search.js"></script><script>var preloader = {
  endLoading: () => {
    document.body.style.overflow = 'auto';
    document.getElementById('loading-box').classList.add("loaded")
  },
  initLoading: () => {
    document.body.style.overflow = '';
    document.getElementById('loading-box').classList.remove("loaded")

  }
}
window.addEventListener('load',preloader.endLoading())</script><div class="js-pjax"></div><script defer="defer" id="fluttering_ribbon" mobile="false" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/canvas-fluttering-ribbon.min.js"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>